# Helper libraries
import datetime
import time
from packaging import version
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.ensemble import RandomForestClassifier
from collections import Counter
import numpy as np
import pandas as pd
from numpy.random import seed
# TensorFlow and tf.keras
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from tensorflow import keras
from tensorflow.keras import models, layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from tensorflow.keras.layers import Dropout, Flatten, Input, Dense
from tensorflow.keras.datasets import fashion_mnist
print("This notebook requires TensorFlow 2.0 or above")
print("TensorFlow version: ", tf.__version__)
assert version.parse(tf.__version__).release[0] >=2
print("Keras version: ", keras.__version__)
%matplotlib inline
def warn(*args, **kwargs):
pass
import warnings
warnings.warn = warn
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
print('train_images:\t{}'.format(train_images.shape))
print('train_labels:\t{}'.format(train_labels.shape))
print('test_images:\t\t{}'.format(test_images.shape))
print('test_labels:\t\t{}'.format(test_labels.shape))
print("First ten labels training dataset:\n {}\n".format(train_labels[0:10]))
print("This output is the numeric label, need to convert to item description")
# from collections import Counter
Counter(train_labels).most_common()
Counter(test_labels).most_common()
fig = plt.figure(figsize = (15, 9))
for i in range(4):
plt.subplot(2, 2, 1+i)
plt.title(train_labels[i])
plt.xticks([])
plt.yticks([])
plt.imshow(train_images[i].reshape(28,28), cmap='binary')
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
fig = plt.figure(figsize = (15, 9))
for i in range(50):
plt.subplot(5, 10, 1+i)
plt.title(class_names[train_labels[i]], fontsize=14)
plt.xticks([])
plt.yticks([])
plt.imshow(train_images[i].reshape(28,28), cmap='binary')
np.set_printoptions(linewidth=np.inf)
print("{}".format(train_images[0]))
train_images = train_images.reshape((60000, 28, 28, 1))
train_images_norm = train_images.astype('float32')/255
test_images = test_images.reshape((10000, 28, 28, 1))
test_images_norm = test_images.astype('float32')/255
train_labels_encoded = to_categorical(train_labels)
test_labels_encoded = to_categorical(test_labels)
def dnnModel(num_layers, hidden_units, hidden_activation, dropout, epochs, optimizer, name):
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Flatten())
for x in range(0,num_layers):
model.add(layers.Dense(units=hidden_units[x], activation=hidden_activation))
if dropout > 0:
model.add(layers.Dropout(dropout))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=epochs,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':name,
'Seed':5,
'Layers':num_layers,
'Hidden Nodes':hidden_units,
'Hidden Activation':hidden_activation,
'Dropout Rate':dropout,
'Epochs':epochs,
'Optimizer':optimizer,
'Time':total_time,
'Test Accuracy':accuracy
}
return (model,results)
def modelInsights(model):
preds = model.predict(test_images_norm)
plot = plt.figure(figsize = (12, 16))
start_index = 0
for i in range(25):
plot = plt.subplot(5, 5, i + 1)
plot = plt.grid(False)
plot = plt.xticks([])
plot = plt.yticks([])
pred = np.argmax(preds[start_index + i])
actual = np.argmax(test_labels_encoded[start_index + i])
col = 'g'
if pred != actual:
col = 'r'
plot = plt.xlabel('i={}\nPredicted={}\nActual={}'.format(start_index + i, class_names[pred], class_names[actual]), color = col)
plot = plt.imshow(test_images[start_index + i].reshape(28,28), cmap='binary')
print('First 25 Predictions')
print('--------------------')
plt.show()
print()
# Get the predicted classes (Test):
pred_test = np.argmax(model.predict(test_images_norm), axis=-1)
conf_mx = tf.math.confusion_matrix(test_labels, pred_test)
print('Confusion Matrix (Testing Set)')
print('------------------------------')
fig, ax = plt.subplots(figsize=(15,15))
ax = sns.heatmap(conf_mx, annot=True, fmt='g')
ax.set_xlabel("Predicted Classes")
ax.set_ylabel("Actual Classes")
ax.set_xticklabels(class_names)
ax.set_yticklabels(class_names, va='center')
plt.show()
print()
for class_index in range(0,10):
i = 0
fig = plt.figure(figsize=(12,6))
fig.suptitle('First 5 {} Images'.format(class_names[class_index]))
for image in np.where(test_labels==class_index)[0][0:10]:
ax = fig.add_subplot(2,5,i+1)
ax.set_xticks([])
ax.set_yticks([])
pred = np.argmax(preds[image])
actual = np.argmax(test_labels_encoded[image])
col = 'g'
if pred != actual:
col = 'r'
ax.set_xlabel('i={}\nPredicted={}\nActual={}'.format(image, class_names[pred], class_names[actual]), color = col)
ax.imshow(test_images[image].reshape(28,28), cmap='binary')
i = i+1
dnn_results = pd.DataFrame(columns=['Model','Seed','Layers','Hidden Nodes','Hidden Activation','Dropout Rate','Epochs','Optimizer','Time','Test Accuracy'])
node_options = [10, 20, 40, 80, 160, 320]
for dropout_option in [0,0.2]:
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp1_'+str(nodes)+'_Model'
else:
folder_name = 'Exp1_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=2,
hidden_units=[nodes,nodes],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp2_'+str(nodes)+'_Model'
else:
folder_name = 'Exp2_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=3,
hidden_units=[nodes,nodes,nodes],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp2b_'+str(nodes)+'_Model'
else:
folder_name = 'Exp2b_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=3,
hidden_units=[nodes,nodes/2,nodes/4],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp2c_'+str(nodes)+'_Model'
else:
folder_name = 'Exp2c_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=3,
hidden_units=[nodes,nodes,nodes/2],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
dnn_results.to_csv('DNN_Results.csv')
dnn_results
dnn_results.to_csv('/content/drive/My Drive/Colab Notebooks/DNN_Results.csv')
exp1_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp1_320_Model')
modelInsights(exp1_best)
exp1_dropout_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp1_320_Model_Dropout')
modelInsights(exp1_dropout_best)
dnn_results['Experiment'] = dnn_results['Model'].apply(lambda x: x.split('_')[0])
dnn_results['N'] = dnn_results['Hidden Nodes'].apply(lambda x: x[0])
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp1']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
exp2_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2_320_Model')
modelInsights(exp2_best)
exp2_dropout_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2_320_Model_Dropout')
modelInsights(exp2_dropout_best)
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp2']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
exp2b_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2b_320_Model')
modelInsights(exp2b_best)
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp2b']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
exp2c_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2c_80_Model')
modelInsights(exp2c_best)
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp2c']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
cnn_results = pd.DataFrame(columns=['Model','Seed','Layers','Filters','Hidden Activation','Dropout Rate','Epochs','Optimizer','Time','Test Accuracy'])
def cnnModel(num_layers, num_filters, hidden_activation, dropout, epochs, optimizer, name):
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
for x in range(0,num_layers):
model.add(layers.Conv2D(filters=num_filters[x], kernel_size=(3,3), strides=(1,1), activation=hidden_activation, input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation=hidden_activation))
if dropout > 0:
model.add(layers.Dropout(dropout))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=epochs,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {'Model':name,
'Seed':5,
'Layers':num_layers,
'Filters':num_filters,
'Hidden Activation':hidden_activation,
'Dropout Rate':dropout,
'Epochs':epochs,
'Optimizer':optimizer,
'Time':total_time,
'Test Accuracy':accuracy
}
print(total_time)
return (model,results)
filter_options = [1, 2, 4, 8, 16, 32]
for dropout_option in [0,0.2]:
# Experiment 3
for filters in filter_options:
print('Running model with {} -> {} filters'.format(filters,filters*2))
if dropout_option == 0:
folder_name = 'Exp3_'+str(filters)+'_Model'
else:
folder_name = 'Exp3_'+str(filters)+'_Model_Dropout'
exp_model = cnnModel(
num_layers=2,
num_filters=[filters,(filters*2)],
hidden_activation='relu',
dropout=dropout_option,
epochs=15,
optimizer='adam',
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
cnn_results = cnn_results.append(exp_model[1],ignore_index=True)
cnn_results.to_csv('/content/drive/My Drive/Colab Notebooks/CNN_Results.csv')
# Experiment 4
for filters in filter_options:
print('Running model with {} -> {} -> {} filters'.format(filters,filters*2,filters*2))
if dropout_option == 0:
folder_name = 'Exp4_'+str(filters)+'_Model'
else:
folder_name = 'Exp4_'+str(filters)+'_Model_Dropout'
exp_model = cnnModel(
num_layers=3,
num_filters=[filters,(filters*2),(filters*4)],
hidden_activation='relu',
dropout=dropout_option,
epochs=15,
optimizer='adam',
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
cnn_results = cnn_results.append(exp_model[1],ignore_index=True)
cnn_results.to_csv('/content/drive/My Drive/Colab Notebooks/CNN_Results.csv')
cnn_results
exp3_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp3_32_Model')
modelInsights(exp3_best)
cnn_results['Experiment'] = cnn_results['Model'].apply(lambda x: x.split('_')[0])
cnn_results['N'] = cnn_results['Filters'].apply(lambda x: x[0])
plot_df = cnn_results[cnn_results['Experiment'] == 'Exp3']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
# Repeat Exp3 stopping at 6 Epochs
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=6,
validation_split=(1/60),
batch_size=100,
verbose=True)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/Exp3_32_Model_6Epochs')
exp4_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp4_32_Model')
modelInsights(exp4_best)
plot_df = cnn_results[cnn_results['Experiment'] == 'Exp4']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
exp5f
modelInsights(exp3_best)
def plot_digits(instances, pos, images_per_row=5, **options):
size = 28
images_per_row = min(len(instances), images_per_row)
images = [instance.reshape(size,size) for instance in instances]
n_rows = (len(instances) - 1) // images_per_row + 1
row_images = []
n_empty = n_rows * images_per_row - len(instances)
images.append(np.zeros((size, size * n_empty)))
for row in range(n_rows):
rimages = images[row * images_per_row : (row + 1) * images_per_row]
row_images.append(np.concatenate(rimages, axis=1))
image = np.concatenate(row_images, axis=0)
pos.imshow(image, cmap = 'binary', **options)
pos.axis("off")
pred_classes = np.argmax(exp3_best.predict(test_images_norm), axis=-1)
cl_a, cl_b = 0, 6
X_aa = test_images_norm[(test_labels == cl_a) & (pred_classes == cl_a)]
X_ab = test_images_norm[(test_labels == cl_a) & (pred_classes == cl_b)]
X_ba = test_images_norm[(test_labels == cl_b) & (pred_classes == cl_a)]
X_bb = test_images_norm[(test_labels == cl_b) & (pred_classes == cl_b)]
plt.figure(figsize=(28,28))
p1 = plt.subplot(221)
p2 = plt.subplot(222)
p3 = plt.subplot(223)
p4 = plt.subplot(224)
plot_digits(X_aa[:25], p1, images_per_row=5);
plot_digits(X_ab[:25], p2, images_per_row=5);
plot_digits(X_ba[:25], p3, images_per_row=5);
plot_digits(X_bb[:25], p4, images_per_row=5);
p1.set_title(f"{class_names[cl_a]}'s classified as {class_names[cl_a]}'s")
p2.set_title(f"{class_names[cl_a]}'s classified as {class_names[cl_b]}'s")
p3.set_title(f"{class_names[cl_b]}'s classified as {class_names[cl_a]}'s")
p4.set_title(f"{class_names[cl_b]}'s classified as {class_names[cl_b]}'s")
plt.show()
# Extracts the outputs of the all the layers
layer_outputs = [layer.output for layer in exp3_best.layers]
# Creates a model that will return these outputs, given the model input:
activation_model = models.Model(inputs=exp3_best.input, outputs=layer_outputs)
layer_names = []
for layer in exp3_best.layers:
layer_names.append(layer.name)
layer_names
from tensorflow.keras.preprocessing import image
(_,_), (test_images, test_labels) = fashion_mnist.load_data()
img = test_images[19]
img_tensor = image.img_to_array(img)
img_tensor = np.expand_dims(img_tensor, axis=0)
# https://www.tensorflow.org/tutorials/keras/classification
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
plt.imshow(img, cmap="binary")
plt.axis('off')
plt.title(class_names[test_labels[40]], fontsize=12)
plt.show()
activations = activation_model.predict(img_tensor)
len(activations)
# These are the names of the layers, so can have them as part of our plot
layer_names = []
for layer in exp3_best.layers:
layer_names.append(layer.name)
images_per_row = 16
# Now let's display our feature maps
for layer_name, layer_activation in zip(layer_names, activations):
if layer_name[0:7] == 'flatten':
break
# This is the number of features in the feature map
n_features = layer_activation.shape[-1]
# The feature map has shape (1, size, size, n_features)
size = layer_activation.shape[1]
# We will tile the activation channels in this matrix
n_cols = n_features // images_per_row
display_grid = np.zeros((size * n_cols, images_per_row * size))
# We'll tile each filter into this big horizontal grid
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,
:, :,
col * images_per_row + row]
# Post-process the feature to make it visually palatable
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255).astype('uint8')
display_grid[col * size : (col + 1) * size,
row * size : (row + 1) * size] = channel_image
# Display the grid
scale = 1. / size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
plt.grid(False)
plt.imshow(display_grid, aspect='auto', cmap='binary')
plt.show()
keras.utils.plot_model(exp3_best, "Experiment3_Model.png", show_shapes=True)
exp_denseSize_results = pd.DataFrame(columns=['Model','Dense Layer Size','Time','Test Accuracy'])
for layer_size in [10, 20, 40, 80, 160, 320, 640]:
seed(5)
tf.random.set_seed(5)
folder_name = 'exp_denseSize_' + str(layer_size)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=layer_size, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':folder_name,
'Dense Layer Size':layer_size,
'Time':total_time,
'Test Accuracy':accuracy
}
exp_denseSize_results = exp_denseSize_results.append(results, ignore_index=True)
exp_denseSize_results.to_csv('/content/drive/My Drive/Colab Notebooks/denseSize_results.csv')
model.save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
exp_denseSize_results
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/addDenseLayer')
exp_denseSize2_results = pd.DataFrame(columns=['Model','Dense Layer Size','Time','Test Accuracy'])
for layer_size in [10, 20, 40, 80, 160, 320, 640]:
seed(5)
tf.random.set_seed(5)
folder_name = 'exp_denseSize2_' + str(layer_size)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=layer_size, activation='relu'))
model.add(layers.Dense(units=layer_size, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':folder_name,
'Dense Layer Size':layer_size,
'Time':total_time,
'Test Accuracy':accuracy
}
exp_denseSize2_results = exp_denseSize2_results.append(results, ignore_index=True)
exp_denseSize2_results.to_csv('/content/drive/My Drive/Colab Notebooks/denseSize2_results.csv')
model.save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
exp_denseSize2_results
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(2,2), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(2,2), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/increaseConvStride')
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=3))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=3))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/increasePoolStride')
exp_batchSize_results = pd.DataFrame(columns=['Model','Dense Layer Size','Time','Test Accuracy'])
for batch in [500, 1000, 5000, 10000, 50000]:
seed(5)
tf.random.set_seed(5)
folder_name = 'exp_batchSize_' + str(batch)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=batch,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':folder_name,
'Batch Size':batch,
'Time':total_time,
'Test Accuracy':accuracy
}
exp_batchSize_results = exp_batchSize_results.append(results, ignore_index=True)
exp_batchSize_results.to_csv('/content/drive/My Drive/Colab Notebooks/batchSize_results.csv')
model.save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
exp_batchSize_results